import scrapy
from scrapy import Request
import re
class NovelspiderSpider(scrapy.Spider):
    name = 'novelspider'
    allowed_domains = ['novel.tingroom.com']
    start_urls = ['http://novel.tingroom.com/']
    gen_url = 'http://novel.tingroom.com'

    def __init__(self):
        self.item = {}

    def parse(self, response):
        category_name = response.xpath("//ul[@class='uul']//a//text()").extract()
        category_href = response.xpath("//ul[@class='uul']//a//@href").extract()
        for category in category_href:
            yield Request(self.gen_url+category,callback=self.parse_category)

    def parse_category(self,response):
        page_href = response.xpath("//div[@class='pages']//a//@href").extract()[0]
        category_code = re.findall(r'\d+',page_href)[0]
        for number in range(1,2):
            href = response.url + "list_"+category_code+"_"+str(number)+".html"
            yield Request(href,callback=self.parse_list)

    def parse_list(self,response):
        # response的根匹配目录xpath开头需要加上//
        list_novel = response.xpath("//div[@class='list']//div[@class='text']")
        #以后的则不需要
        href_novel = list_novel.xpath("h6[@class='yuyu']//a//@href").extract()

        profile = list_novel.xpath("p[@class='intro11']//text()").extract()

        while "简介：" in profile:
            profile.remove("简介：")
        for i in range(0,len(href_novel)):
            href = self.gen_url+href_novel[i]

            yield Request(href,callback=self.parse_novel)

    def parse_novel(self,response):

        #一次性装如item就没有出问题
        category_dic = {"jingdian":"classic","shuangyu":"languages","mingren":"celebrity","lizhi":"inspiring","duanpian":"short article","kehuan":"science fiction","ertong":"children","zongjiao":"religion"}
        img = response.xpath("//div[@class='clearfix hslice']//img//@src").extract_first()
        title = response.xpath("//div[@class='entry-content']//h1//a//text()").extract()
        download_href =self.gen_url+response.xpath("//div[@style='text-align:center;']//a//@href").extract()[1]
        profile = response.xpath("//div[@id='infoShort']//p//text()").extract_first()
        novel_id = re.findall(r'\d+',response.url)
        info = response.xpath("//div[@class='entry-content']//span//a//text()").extract()
        i = response.xpath("//div[@class='entry-content']//span//a[1]//@href").extract_first()
        # response.xpath("//ol[@]")
        if 'writer' in i:
            self.item["author"] = info[0]
        else:
            self.item["author"] = "未知"
        href = response.url
        for key in category_dic:
            if key in href:
                self.item["category"] = category_dic[key]
                break
        if img=='/template/default/tingnovel/images/defaultpic.gif':
            self.item["img"] =self.gen_url+img
        else:
            self.item["img"] = img
        self.item["title"] = title[0]
        self.item["download_href"] = download_href
        self.item["profile"] = profile
        self.item["novel_id"] = novel_id[0]
        yield self.item

